{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "96561d5f",
   "metadata": {},
   "source": [
    "# Dataprep EDA for Spark dataframe with ray"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "076de06e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import ray\n",
    "import raydp\n",
    "from dataprep.eda import create_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b3b8fad",
   "metadata": {},
   "outputs": [],
   "source": [
    "ray.init()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a1ca9275",
   "metadata": {},
   "outputs": [],
   "source": [
    "spark = raydp.init_spark(\n",
    "    app_name=\"eda\",\n",
    "    num_executors=2,\n",
    "    executor_cores=2,\n",
    "    executor_memory=\"1G\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a1ef96be",
   "metadata": {},
   "outputs": [],
   "source": [
    "spark_df = spark.read.csv(\"file:///dataprep/datasets/data/titanic.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "df11edd8",
   "metadata": {},
   "outputs": [],
   "source": [
    "ray_df = ray.data.from_spark(spark_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4895c49",
   "metadata": {},
   "outputs": [],
   "source": [
    "dask_df = ray_df.to_dask()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e805cfbb",
   "metadata": {},
   "outputs": [],
   "source": [
    "create_report(dask_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b203d0cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "ray.shutdown()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  },
  "nbsphinx": {
   "execute": "never"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}